﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Common;
using Dal;
using System.Text.RegularExpressions;
using Stemmer;
using Site_Extractor;
using Rss;
using System.Threading;
using Indexing.Properties;
using System.IO;
using System.Net;
using Crawling;
using Amib.Threading;

namespace Indexing
{

    public class Indexer
    {
        public const int MAX_TIMEOUT_PASSES = 1;
        string dbConnectionString;
        public Catalog Catalog;
        //PorterStemmer Stemmer = new PorterStemmer();
        StopWordsHandler StopWordsHandler = new StopWordsHandler();
        object lockNumOfWorkingThreads = new object();
        object lockObject = new object();
        object lockVisitedLinks = new object();
        int numberOfWorkingThreads; //if 0, and linkstocrawlqueue.count==0, then no more links to crawl.
        //MiscUtil.Threading.CustomThreadPool ThreadPool = new MiscUtil.Threading.CustomThreadPool();
        SmartThreadPool ThreadPool;
        List<string> VisitedLinks = new List<string>();
        List<string> TimeoutList = new List<string>();
        int TimeoutPasses = 0;
        public delegate void WorkItemDelegate(object obj);

        public Indexer(Catalog catalog, string DbConnectionString, string prefixFilePath, int MaxThreads)
        {
            dbConnectionString = DbConnectionString;
            Catalog = catalog;
            ThreadPool = new SmartThreadPool(0, MaxThreads, MaxThreads);
           
           
        }

        private bool IsDocumentEmpty(Document Document)
        {
            bool IsEmpty = true;
            for (int i = 0; i < Document.NumberOfFields; i++)
            {
                if (Document.Fields[i].Text != string.Empty) return false;
            }
            return IsEmpty;

        }

        /// <summary>
        /// Indexes document to DB and Catalog DS , where needed.
        /// </summary>
        /// <param name="Document"></param>
        public void IndexDocument(Document Document)
        {
            if (null == Document || IsDocumentEmpty(Document))
                return;
            long documentID;
            //Index to DB
            bool IsNewDocument = IndexDocumentToDB(Document, out documentID);
            Document.Id = documentID;
            //Index to Catalog
            if (IsNewDocument)
            {
                //lock (lockObject)
                //{
                IndexDocumentToCatalog(Document);
                // }
            }
        }


        /// <summary>
        /// Indexes the document to the Catalog Data Structure if it's not already in there.
        /// </summary>
        /// <param name="Document"></param>
        private void IndexDocumentToCatalog(Document Document)
        {
            //If document was already indexed OR document is empty, abort.
            if (null != Catalog.GetDocumentByID(Document.Id))
                return;


            string[] TextArray;
            for (int i = 0; i < Document.NumberOfFields; i++)
            {
                TextArray = QueryParser.GetParsedPhraseArray(Document.Fields[i].Text, true);
                Document.Fields[i].Length = TextArray.Length;
                for (int j = 0; j < TextArray.Length; j++)
                {
                    WordPos WordPos = new WordPos();
                    WordPos.Position = j + 1;
                    WordPos.Section = (Document.Section)i;
                    lock (lockObject)
                    Catalog.AddWord(new Word(TextArray[j]), Document.Id, WordPos);

                }
                Catalog.SumOfFieldLength[i] += Document.Fields[i].Length;
            }

            //Statistics for ranking.
            Catalog.NumberOfDocumentsIndexed++;
            lock (lockObject)
            Catalog.AddDocument(Document);
        }

        /// <summary>
        /// indexes a Document in DB, out value is the doc's unique ID.
        /// </summary>
        /// <param name="Document"></param>
        /// <param name="DocumentID"></param>
        /// <returns>if document was alreay in DB returns false, else true.</returns>
        public bool IndexDocumentToDB(Document Document, out long DocumentID) //Return document unique ID
        {

            DatabaseAbstractionLayer dal = new DatabaseAbstractionLayer(dbConnectionString);
            Document doc = dal.GetDocumentByUrlCommentNumber(Document.Url, Document.CommentNumber);
            if (doc == null)
            {

                DocumentID = dal.InsertDocument(Document);
                return true;
            }

            DocumentID = doc.Id;
            return false;
        }

        //Loads all documents from DB to catalog.
        public void LoadDbToCatalog()
        {
            long maxDocId, minDocId;
            DatabaseAbstractionLayer layer = new DatabaseAbstractionLayer(dbConnectionString);
            Document doc;
            maxDocId = layer.GetMaxDocId();
            minDocId = layer.GetMinDocId();
            for (long i = minDocId; i <= maxDocId; i++)
            {
                doc = layer.GetDocumentByID(i);
                if (doc == null) continue;
                IndexDocumentToCatalog(doc);
            }
        }

      
        public Uri GetUri(string domain, string url)
        {
            if (url.StartsWith(@"/"))
                url = "http://" + domain + url;
            if (!url.StartsWith(@"http://"))
                url = @"http://" + url;
            return new Uri(url);
        }

        /// <summary>
        /// Indexes All forums on the site, from a specific date.
        /// This method uses Site's domains to know which Forum Extractor to use.
        /// </summary>
        /// <param name="site"></param>
        /// <param name="FromDate"></param>
        public void IndexAllForums(Site site, DateTime FromDate)
        {
            /*ForumExtractorFactory fef = new ForumExtractorFactory();
            IForumExtractor ForumExtractor = fef.GetForumExtractorByDomain(site.Domains[1]);
            long[] forumIds = ForumExtractor.GetAllForumIds();
            for (int i = 0; i < forumIds.Length; i++)
            {

                IndexForum(ForumExtractor.ExtractForum(forumIds[i]), FromDate);
            }*/
        }

        /// <summary>
        /// Indexes all stories on the site. Uses Site's CrawlUri list to know 
        /// where to crawl for the stories.
        /// </summary>
        /// <param name="Site"></param>
        /// <param name="domain"></param>
        /// <param name="FromDate"></param>
        public void IndexStories(Site Site, string domain, DateTime FromDate)
        {
            IndexStories(Site, domain, FromDate, Settings.Default.IndexMultithreaded);
        }



        public void IndexForums(Site Site, DateTime FromDate)
        {
            if (null == Site || null == Site[RuleSet.Forum] || null == Site[RuleSet.Forum].TalkbackRule)
                return;
             RuleSet UriRuleSet = Site[RuleSet.Forum]; // Forum rules
             foreach (string Uri in UriRuleSet.ContentUris)
             {
                 HandleUrl(new StoryIndexArguments(Site.SiteNames.SiteName, new Uri(Uri), Site[RuleSet.Forum], FromDate, UriRuleSet.CrawlDepth) as object);
             }
             ThreadPool.WaitForIdle();
            
        }


        public void IndexBlogs(Site Site, DateTime FromDate)
        {
            
            if (null == Site || null == Site[RuleSet.Blog])// || domain == String.Empty)
                return;
            RuleSet UriRuleSet = Site[RuleSet.Blog]; //Get the rules for getting uris
            foreach (string Uri in UriRuleSet.ContentUris)
            {
                HandleUrl(new StoryIndexArguments(Site.SiteNames.SiteName, new Uri(Uri), Site[RuleSet.Blog], FromDate, UriRuleSet.CrawlDepth) as object);
            }
            ThreadPool.WaitForIdle();

           
        }

        public void IndexStories(Site Site, string domain, DateTime FromDate, bool multiThreaded)
        {
               if (null == Site || null == Site[RuleSet.Story])// || domain == String.Empty)
                return;
            RuleSet UriRuleSet = Site[RuleSet.Story]; //Get the rules for getting uris
            
            //List<string> siteLinks = new List<string>();
            //Crawler Crawler = new Crawler();
            foreach (string Uri in UriRuleSet.ContentUris)
            {
                HandleUrl(new StoryIndexArguments(Site.SiteNames.SiteName, new Uri(Uri), Site[RuleSet.Story], FromDate, UriRuleSet.CrawlDepth) as object);
            }
            ThreadPool.WaitForIdle();
        }

        public void RetryTimedOutJobs(Site Site, DateTime FromDate)
        {
            ///Another pass or two for timed out links
            while (TimeoutPasses < MAX_TIMEOUT_PASSES)
            {
                TimeoutPasses++;
                while (TimeoutList.Count > 0)
                {
                    HandleUrl(new StoryIndexArguments( Site.SiteNames.SiteName, new Uri(TimeoutList[0]), Site[RuleSet.Story], FromDate, Site[RuleSet.Story].CrawlDepth) as object);
                    TimeoutList.RemoveAt(0);
                }
                JobManager.WaitForAll();
                if (TimeoutList.Count == 0)
                    return;
            }
            
        }


        public bool IsNotXml(Uri uri)
        {
            DatabaseAbstractionLayer dal = new DatabaseAbstractionLayer("");
            return dal.IsNotXml(uri);
        }

        public bool IsXmlUri(List<string> xmlUris, Uri uri)
        {
            foreach (string xu in xmlUris)
                if (uri.ToString().ToLower().Contains(xu.ToLower()))
                    return true;
            return false;
        }

        
        public void SetUrlData(Uri uri, DateTime lastDownloadDate, bool isXml, string etag)
        {
            DatabaseAbstractionLayer dal = new DatabaseAbstractionLayer("");
            dal.SetUrlData(uri, lastDownloadDate, isXml, etag );
        }
        
        /// <summary>
        /// Handle URI according to what it contains. RSS feed, or story to index.
        /// </summary>
        /// <param name="storyIndexArguments"></param>
        public object HandleUrl(object storyIndexArguments)
        {

            if (!(storyIndexArguments is StoryIndexArguments)) return new object();
            
            StoryIndexArguments sia = storyIndexArguments as StoryIndexArguments;
			int depth = sia.Depth;
			//if (depth < 0) return;
            lock (lockVisitedLinks)
            {
                if (VisitedLinks.Contains(sia.Uri.ToString()) && depth < 0)
                    return new object();
                VisitedLinks.Add(sia.Uri.ToString());
            }

            try
            {
                /*
                bool isXMl = false;
                if (IsXmlUri(sia.RuleSet.XmlUris, sia.Uri))
                    isXMl = true;
                else
                {
                    if (HttpHelper.HttpDownloader.isXML(sia.Uri))
                        isXMl = true;
                }
                */
                if (HttpHelper.HttpDownloader.isXML(sia.Uri))
                {
                    //SetUrlData(sia.Uri, DateTime.MinValue, true);
                    //Console.WriteLine("XML to parse : " + sia.Uri);
                    ThreadPool.QueueWorkItem(new WorkItemCallback(AddRssFeedLinksToQueue), sia);
                }
                 
                else
                {
                    SetUrlData(sia.Uri, DateTime.MinValue, false, "");
                    if (depth >= 0)
                    {
                        // get all links, run handleUrl on each

                        Crawler c = new Crawler();
                        var links = c.Crawl(sia.Uri.OriginalString, 0, false);
                        foreach (string link in links)
                        {
                            var nextSia = new StoryIndexArguments(sia.SiteName, new Uri(link), sia.RuleSet, sia.MinDate, sia.Depth);
                            nextSia.Depth = sia.Depth - 1;
                            ThreadPool.QueueWorkItem(new WorkItemCallback(HandleUrl), nextSia);
                        }
                    }
                   
                        //Console.WriteLine("Url to Index : " + sia.Uri);
                    if (sia.RuleSet.Name == RuleSet.Forum)
                        ThreadPool.QueueWorkItem(new WorkItemCallback(IndexForum), sia);
                    else
                        ThreadPool.QueueWorkItem(new WorkItemCallback(IndexStory), sia);

				}                
            }
            catch (Exception e)
            {
                Console.WriteLine("An error Has Occured in HandleURL :  " + e.Message);
            }
            return new object();

        }


        public bool WasIndexed(string Url, long commentNum)
        {
            DatabaseAbstractionLayer dal = new DatabaseAbstractionLayer("");
            return dal.IsDocumentExist(Url, commentNum);
        }

        public object IndexForum(object sia)
        {
            StoryIndexArguments s = sia as StoryIndexArguments;
            DateTime FromDate = s.MinDate;
            RuleSet RuleSet = s.RuleSet;
            string SiteName = s.SiteName;
            Uri Uri = s.Uri;
            string HtmlPage;


            if (!SiteExtractor.IsContent(Uri, RuleSet.IdentifyingUris))
                return new object();
            try
            {
                HtmlPage = HttpHelper.HttpDownloader.DownloadPage(Uri, RuleSet.Encoding, false, true);
            }
            catch (Exception e)
            {
                AddTimedOutUrlToList(Uri, e);
                Console.WriteLine("Exception: {0}", e.Message);
                return new object();
            }
            List<Talkback> ForumMessages = TalkBackExtractor.ExtractTalkBacks(SiteName,
                RuleSet, Uri, HtmlPage, DateTime.MinValue, FromDate);

            foreach (Talkback fm in ForumMessages)
            {
                fm.SiteName = SiteName;
                fm.Uri = Uri;
                IndexTalkback(new TalkbackIndexArguments(fm, new Uri(Uri.ToString()), RuleSet, FromDate));
            }
            return new object();

        }

        /// <summary>
        /// Indexes a forum to DB + Catalog DS.
        /// </summary>
        /// <param name="forum"></param>
        /// <param name="FromDate"></param>
        public void IndexForum(Forum forum, DateTime FromDate)
        {
            for (int i = 0; i < forum.Messages.Count; i++)
            {
                Document ForumDocument = CreateForumMessageDocument(forum.Messages[i], i, forum.Url);
                IndexDocument(ForumDocument);
            }
        }

        public void IndexTalkBacks(Uri Uri, List<Talkback> Talkbacks)
        {
            foreach (Talkback tb in Talkbacks)
            {
                Document tbdoc = CreateTalkbackDocument(tb, Uri);
                IndexDocument(tbdoc);
            }
        }

        public object AddRssFeedLinksToQueue(object storyIndexArguments)
        {
            if (!(storyIndexArguments is StoryIndexArguments)) return new object();
            StoryIndexArguments sia = storyIndexArguments as StoryIndexArguments;
            string url = sia.Uri.ToString();
            try
            {
                string XmlPage = HttpHelper.HttpDownloader.DownloadPage(sia.Uri, Encoding.Default.WebName, true, true);
                Rss.RssReader RssReader = new Rss.RssReader(new StringReader(XmlPage));
                while (true)
                {
                  
                    RssElement element = RssReader.Read();
                    if (null == element) throw new Exception();
                    if (!(element is RssItem)) continue;
                    var item = element as RssItem;
                    sia.Uri = item.Link;

                    lock (lockVisitedLinks)
                    {
                        if (VisitedLinks.Contains(item.Link.ToString()))
                            continue;
                        VisitedLinks.Add(item.Link.ToString());
                    }
                    ThreadPool.QueueWorkItem(new WorkItemCallback(IndexStory), new StoryIndexArguments(sia) as object);
                   
                }
                
            }


            catch (Exception e)
            {
                Console.WriteLine("Exception: {0}", e.Message);
                if (e is System.Net.WebException)
                    Console.Write(e);
            }
            return new object();
        }


        public void AddTimedOutUrlToList(Uri uri, Exception e)
        {
            if (e is WebException)
            {
                if (((WebException)e).Status == WebExceptionStatus.Timeout)
                {
                    TimeoutList.Add(new Uri(uri.ToString()).ToString());
                }
            }
        }

        public object IndexStory(object storyIndexArguments)
        {
            if (!(storyIndexArguments is StoryIndexArguments))
                return new object();
            IndexStory(storyIndexArguments as StoryIndexArguments);
            return new object();
        }


        /// <summary>
        /// Indexes a story to DB + Catalog DS
        /// </summary>
        /// <param name="story"></param>
        /// <param name="site"></param>
        /// <param name="uri"></param>
        /// <param name="FromDate"></param>
        public void IndexStory(StoryIndexArguments storyIndexArguments)
        {
            //Console.WriteLine("Indexing Story : " + storyIndexArguments.Uri);
            Uri Uri = storyIndexArguments.Uri;
            RuleSet ruleSet = storyIndexArguments.RuleSet;
            DateTime FromDate = storyIndexArguments.MinDate;
            string siteName = storyIndexArguments.SiteName;
            Story story = new Story();
            DateTime defaultTalkbackDate = DateTime.MinValue;
            if (null == Uri || null == ruleSet || !SiteExtractor.IsContent(Uri,
                  ruleSet.IdentifyingUris))
                return;

            string HtmlPage = string.Empty;

            //Check if story doesn't exist in DB, or story contains talkbacks (need to d/l article)
            DatabaseAbstractionLayer dal = new DatabaseAbstractionLayer(dbConnectionString);
            Document doc = dal.GetDocumentByUrlCommentNumber(Uri, Story.StoryCommentNumber);
            bool DocumentExists = !( doc == null);
            if (!DocumentExists ||
              ruleSet.TalkbackRule.ArticleContainsTalkbacks || ruleSet.TalkbackRule.UrlContainsArticleID)
            {
                try
                {
                    HtmlPage = HttpHelper.HttpDownloader.DownloadPage(Uri, ruleSet.Encoding, false, true);
                }
                catch (Exception e)
                {
                    AddTimedOutUrlToList(Uri, e);
                    Console.WriteLine("Exception: {0}", e.Message);
                    return;
                }

            }
            try
            {
                //Index document.
                if (!DocumentExists)
                {
                    //Story was not yet indexed in DB.
                    story = SiteExtractor.ExtractStory(siteName, ruleSet,
                      Uri, HtmlPage);
                    if (!(SiteExtractor.IsStory(story)))
                    {
                       // story = HtmlDownloader.GetStory(Site, Uri);
                          // if (!(SiteExtractor.IsStory(story)))
                               return;
                    }
                      
                    if (story.DateTime >= FromDate)
                    {
                        Document StoryDocument = CreateStoryDocument( ruleSet.GetDocType(), story, Uri);
                        IndexDocument(StoryDocument);
                        defaultTalkbackDate = story.DateTime;
                    }
                }
                else
                    defaultTalkbackDate = doc.dateTime;
                //Index talkbacks if needed.
                //if (Site.TalkbackRule.ArticleContainsTalkbacks)
                List<Talkback> Tbs = TalkBackExtractor.ExtractTalkBacks(siteName, ruleSet, Uri, HtmlPage, defaultTalkbackDate, FromDate);
                foreach (Talkback tb in Tbs)
                {
                    ThreadPool.QueueWorkItem(new WorkItemCallback(IndexTalkback), new TalkbackIndexArguments(tb, Uri, ruleSet, FromDate));
                }
                Console.WriteLine("Indexed Story : " + Uri);
            }
            catch (Exception e)
            {
                AddTimedOutUrlToList(Uri, e);
                Console.WriteLine("Index Story - " + Uri.ToString() + " , Exception: {0}", e.Message);
                //throw e;
                
            }
        }

        public object IndexTalkback(object talkbackIndexArguments)
        {
            if (!(talkbackIndexArguments is TalkbackIndexArguments))
                return new object();
            IndexTalkback(talkbackIndexArguments as TalkbackIndexArguments);
            return new object();
        }

        public void IndexTalkback(TalkbackIndexArguments talkbackIndexArguments)
        {
            Talkback talkback = talkbackIndexArguments.Talkback;
            Uri uri = talkbackIndexArguments.Uri;
            RuleSet ruleSet = talkbackIndexArguments.RuleSet;
            DateTime minDate = talkbackIndexArguments.MinDate;

            if (talkback.DateTime < minDate)
                return;
            if (WasIndexed(uri.ToString(), talkback.ID))
                return;
            if (talkback.ContentUri != null)
            {
                talkback.Text = TalkBackExtractor.GetTalkbackContentFromUrl(ruleSet, talkback.ContentUri,
                    ruleSet.TalkbackRule.UrlContainsOneTalkbackContent.TextExtractExpression,
                    TalkbackRule.ContentToken);
            }
            Document tbdoc;
            if (RuleSet.Forum == ruleSet.Name)
            {
                tbdoc = CreateForumMessageDocument(new ForumMessage(talkback), talkback.ID, talkback.Uri.ToString());

                IndexDocument(tbdoc);
                Console.WriteLine("Indexed ForumMessage For " + uri + " Num : " + tbdoc.Id.ToString());
            }
            else
            {
                tbdoc = CreateTalkbackDocument(talkback, uri);
                IndexDocument(tbdoc);
                Console.WriteLine("Indexed Talkback For " + uri + " Num : " + tbdoc.Id.ToString());
        
            }
           
              }


        public Document CreateStoryDocument(Document.Type Type, Story Story, Uri Uri)
        {
            Document Document = new Document(1, Story.DateTime, Uri, 0, 0, Type, 0, Story.SiteName);
            Document.Fields[(int)Document.Section.Author] = new Field("", Document.Section.Author);
            Document.Fields[(int)Document.Section.Title] = new Field(Story.Title, Document.Section.Title);
            Document.Fields[(int)Document.Section.Summary] = new Field(Story.Summary, Document.Section.Summary);
            Document.Fields[(int)Document.Section.Text] = new Field(Story.Text, Document.Section.Text);
            return Document;
        }

        public Document CreateForumMessageDocument(ForumMessage ForumMessage, long MessageNumber, string Url)
        {

            Document Document = new Document(1, ForumMessage.Date, new Uri(Url), ForumMessage.UniqueId, 0, Document.Type.ForumMessage, 0, ForumMessage.SiteName);
            Document.Fields[(int)Document.Section.Author] = new Field("", Document.Section.Author);
            Document.Fields[(int)Document.Section.Title] = new Field(ForumMessage.Topic, Document.Section.Title);
            Document.Fields[(int)Document.Section.Summary] = new Field("", Document.Section.Summary);
            Document.Fields[(int)Document.Section.Text] = new Field(ForumMessage.Text, Document.Section.Text);
            return Document;
        }

        public Document CreateTalkbackDocument(Talkback Talkback, Uri Url)
        {

            Document Document = new Document(1, Talkback.DateTime, Url, Talkback.ID, Talkback.Number, Document.Type.Talkback, 0, Talkback.SiteName);
            Document.Fields[(int)Document.Section.Author] = new Field("", Document.Section.Author);
            Document.Fields[(int)Document.Section.Title] = new Field(Talkback.Title, Document.Section.Title);
            Document.Fields[(int)Document.Section.Summary] = new Field("", Document.Section.Summary);
            Document.Fields[(int)Document.Section.Text] = new Field(Talkback.Text, Document.Section.Text);
            return Document;
        }

		public void IndexForum(GenericForum f, DateTime from)
		{
            JobManager.SetMaxThreads(2, 2);
			try
			{
				var links = f.GetAllForumLinks();
				foreach (string link in links)
				{
                    ThreadPool.QueueWorkItem
                        (new WorkItemCallback(IndexForumLink), new object[] { f, from, link });
				}
			}
			catch (Exception ex) { Console.WriteLine(ex.Message); }
		}
		private object IndexForumLink(object o)
		{
			try
			{
				object[] objs = o as object[];
				var f = objs[0] as GenericForum;
				var from = (DateTime)objs[1];
				var link = objs[2] as string;

				var msgInfos = f.GetAllMessageLinks(link);
				foreach (ForumMessageInfo msgInfo in msgInfos)
				{
					if (msgInfo.Date < from) continue;
					JobManager.AddJob(IndexForumMessage, new object[] {f, msgInfo });
				}
			}
			catch (Exception ex) { Console.WriteLine(ex.Message); }
            return new object();
		}
		private void IndexForumMessage(object o)
		{
			try
			{
				object[] objs = o as object[];
				var f = objs[0] as GenericForum;
				var info = objs[1] as ForumMessageInfo;
				var msg = f.GetMessage(info.Link);
				msg.SiteName = f.Name;

				IndexDocument(CreateForumMessageDocument(msg, msg.UniqueId, info.OriginalLink));
                Console.WriteLine("Indexed Forum Message : " + f.Name);
			}
			catch (Exception ex) { Console.WriteLine(ex.Message); }
		}
	}
}
